Annotations#

Hide imports
%load_ext autoreload
%autoreload 2

import os

import dimcat as dc
import ms3
import plotly.express as px
from dimcat import groupers, plotting

import utils
Hide source
RESULTS_PATH = os.path.abspath(os.path.join(utils.OUTPUT_FOLDER, "overview"))
os.makedirs(RESULTS_PATH, exist_ok=True)


def make_output_path(
    filename: str,
    extension=None,
    path=RESULTS_PATH,
) -> str:
    return utils.make_output_path(filename=filename, extension=extension, path=path)


def save_figure_as(
    fig, filename, formats=("png", "pdf"), directory=RESULTS_PATH, **kwargs
):
    if formats is not None:
        for fmt in formats:
            plotting.write_image(fig, filename, directory, format=fmt, **kwargs)
    else:
        plotting.write_image(fig, filename, directory, **kwargs)

Loading data

Hide source
D = utils.get_dataset("poulenc_mouvements_perpetuels", corpus_release="v2.4")
package = D.inputs.get_package()
package_info = package._package.custom
git_tag = package_info.get("git_tag")
utils.print_heading("Data and software versions")
print("Francis Poulenc – Mouvements Perpetuels version v2.4")
print(f"Datapackage '{package.package_name}' @ {git_tag}")
print(f"dimcat version {dc.__version__}\n")
D
Data and software versions
--------------------------

Francis Poulenc – Mouvements Perpetuels version v2.4
Datapackage 'poulenc_mouvements_perpetuels' @ v2.4
dimcat version 3.4.0
Dataset
=======
{'inputs': {'basepath': None,
            'packages': {'poulenc_mouvements_perpetuels': ["'poulenc_mouvements_perpetuels.measures' "
                                                           '(MuseScoreFacetName.MuseScoreMeasures)',
                                                           "'poulenc_mouvements_perpetuels.notes' "
                                                           '(MuseScoreFacetName.MuseScoreNotes)',
                                                           "'poulenc_mouvements_perpetuels.expanded' "
                                                           '(MuseScoreFacetName.MuseScoreHarmonies)',
                                                           "'poulenc_mouvements_perpetuels.chords' "
                                                           '(MuseScoreFacetName.MuseScoreChords)',
                                                           "'poulenc_mouvements_perpetuels.metadata' "
                                                           '(FeatureName.Metadata)']}},
 'outputs': {'basepath': None, 'packages': {}},
 'pipeline': []}
filtered_D = D.apply_step("HasHarmonyLabelsFilter")
all_metadata = filtered_D.get_metadata()
assert len(all_metadata) > 0, "No pieces selected for analysis."
chronological_corpus_names = all_metadata.get_corpus_names()

DCML harmony labels#

Hide source
all_annotations = filtered_D.get_feature("DcmlAnnotations")
is_annotated_mask = all_metadata.label_count > 0
is_annotated_index = all_metadata.index[is_annotated_mask]
annotated_notes = filtered_D.get_feature("notes").subselect(is_annotated_index)
print(f"The annotated pieces have {len(annotated_notes)} notes.")
The annotated pieces have 1563 notes.
all_chords = filtered_D.get_feature("harmonylabels")
print(
    f"{len(all_annotations)} annotations, of which {len(all_chords)} are harmony labels."
)
277 annotations, of which 268 are harmony labels.

Harmony labels#

Unigrams#

For computing unigram statistics, the tokens need to be grouped by their occurrence within a major or a minor key because this changes their meaning. To that aim, the annotated corpus needs to be sliced into contiguous localkey segments which are then grouped into a major (is_minor=False) and a minor group.

root_durations = (
    all_chords[all_chords.root.between(-5, 6)]
    .groupby(["root", "chord_type"])
    .duration_qb.sum()
)
# sort by stacked bar length:
# root_durations = root_durations.sort_values(key=lambda S: S.index.get_level_values(0).map(S.groupby(level=0).sum()),
# ascending=False)
bar_data = root_durations.reset_index()
bar_data.root = bar_data.root.map(ms3.fifths2iv)
fig = px.bar(
    bar_data,
    x="root",
    y="duration_qb",
    color="chord_type",
    title="Distribution of chord types over chord roots",
    labels=dict(
        root="Chord root expressed as interval above the local (or secondary) tonic",
        duration_qb="duration in quarter notes",
        chord_type="chord type",
    ),
)
fig.update_layout(**utils.STD_LAYOUT)
save_figure_as(fig, "chord_type_distribution_over_scale_degrees_absolute_stacked_bars")
fig.show()
relative_roots = all_chords[
    ["numeral", "duration_qb", "relativeroot", "localkey_is_minor", "chord_type"]
].copy()
relative_roots["relativeroot_resolved"] = ms3.transform(
    relative_roots, ms3.resolve_relative_keys, ["relativeroot", "localkey_is_minor"]
)
has_rel = relative_roots.relativeroot_resolved.notna()
relative_roots.loc[has_rel, "localkey_is_minor"] = relative_roots.loc[
    has_rel, "relativeroot_resolved"
].str.islower()
relative_roots["root"] = ms3.transform(
    relative_roots, ms3.roman_numeral2fifths, ["numeral", "localkey_is_minor"]
)
chord_type_frequency = all_chords.chord_type.value_counts()
replace_rare = ms3.map_dict(
    {t: "other" for t in chord_type_frequency[chord_type_frequency < 500].index}
)
relative_roots["type_reduced"] = relative_roots.chord_type.map(replace_rare)
# is_special = relative_roots.chord_type.isin(('It', 'Ger', 'Fr'))
# relative_roots.loc[is_special, 'root'] = -4
root_durations = (
    relative_roots.groupby(["root", "type_reduced"])
    .duration_qb.sum()
    .sort_values(ascending=False)
)
bar_data = root_durations.reset_index()
bar_data.root = bar_data.root.map(ms3.fifths2iv)
root_order = (
    bar_data.groupby("root")
    .duration_qb.sum()
    .sort_values(ascending=False)
    .index.to_list()
)
fig = px.bar(
    bar_data,
    x="root",
    y="duration_qb",
    color="type_reduced",
    barmode="group",
    log_y=True,
    color_discrete_map=utils.TYPE_COLORS,
    category_orders=dict(
        root=root_order,
        type_reduced=relative_roots.type_reduced.value_counts().index.to_list(),
    ),
    labels=dict(
        root="intervallic difference between chord root to the local or secondary tonic",
        duration_qb="duration in quarter notes",
        type_reduced="chord type",
    ),
    width=1000,
    height=400,
)
fig.update_layout(
    **utils.STD_LAYOUT,
    legend=dict(
        orientation="h",
        xanchor="right",
        x=1,
        y=1,
    ),
)
save_figure_as(fig, "chord_type_distribution_over_scale_degrees_absolute_grouped_bars")
fig.show()
print(
    f"Reduced to {len(set(bar_data.iloc[:,:2].itertuples(index=False, name=None)))} types. "
    f"Paper cites the sum of types in major and types in minor (see below), treating them as distinct."
)
Reduced to 6 types. Paper cites the sum of types in major and types in minor (see below), treating them as distinct.
dim_or_aug = bar_data[
    bar_data.root.str.startswith("a") | bar_data.root.str.startswith("d")
].duration_qb.sum()
complete = bar_data.duration_qb.sum()
print(
    f"On diminished or augmented scale degrees: {dim_or_aug} / {complete} = {dim_or_aug / complete}"
)
On diminished or augmented scale degrees: 0.0 / 313.875 = 0.0
chords_by_mode = groupers.ModeGrouper().process(all_chords)
chords_by_mode.format = "scale_degree"

Whole dataset#

unigram_proportions = chords_by_mode.get_default_analysis()
unigram_proportions.make_ranking_table()
mode major minor
chord_and_mode scale_degrees duration_qb proportion proportion_% chord_and_mode scale_degrees duration_qb proportion proportion_%
rank
1 ii7(9), major (2, 4, 6, 1) 20.5 0.080550 8.06 % iv(96), minor (4, 6, 2) 7.000 0.117895 11.79 %
2 I, major (1, 3, 5) 18.0 0.070727 7.07 % i2, minor (7, 1, 3, 5) 5.000 0.084211 8.42 %
3 iii(b9), major (3, 5, 7) 16.0 0.062868 6.29 % i, minor (1, 3, 5) 5.000 0.084211 8.42 %
4 IVM2, major (3, 4, 6, 1) 14.0 0.055010 5.5 % i64(13), minor (5, 1, 3) 4.000 0.067368 6.74 %
5 IM7, major (1, 3, 5, 7) 13.0 0.051081 5.11 % v, minor (5, 7, 2) 4.000 0.067368 6.74 %
6 V(^2), major (5, 6, 2) 11.0 0.043222 4.32 % ii%65, minor (4, 6, 1, 2) 4.000 0.067368 6.74 %
7 V7(^2), major (5, 6, 2, 4) 9.0 0.035363 3.54 % i(9b6), minor (1, 3, b6) 4.000 0.067368 6.74 %
8 V(96), major (5, 7, 3) 8.0 0.031434 3.14 % ii%43, minor (6, 1, 2, 4) 4.000 0.067368 6.74 %
9 ii7, major (2, 4, 6, 1) 8.0 0.031434 3.14 % i43, minor (5, 7, 1, 3) 3.000 0.050526 5.05 %
10 V(6), major (5, 7, 3) 7.0 0.027505 2.75 % V7(9b6)/III, minor (7, 2, b5, 6) 2.375 0.040000 4.0 %
11 ii, major (2, 4, 6) 6.5 0.025540 2.55 % iio6, minor (4, 6, 2) 2.000 0.033684 3.37 %
12 IM7(9), major (1, 3, 5, 7) 6.0 0.023576 2.36 % V(9)/III, minor (7, 2, 4) 2.000 0.033684 3.37 %
13 i7, major (1, b3, 5, b7) 6.0 0.023576 2.36 % V(96)/III, minor (7, 2, 5) 2.000 0.033684 3.37 %
14 V(6+4), major (5, 7, 3) 6.0 0.023576 2.36 % i(96), minor (1, 3, 6) 2.000 0.033684 3.37 %
15 v(119), major (5, b7, 2) 5.0 0.019646 1.96 % i(9), minor (1, 3, 5) 2.000 0.033684 3.37 %
16 V7(96), major (5, 7, 3, 4) 4.5 0.017682 1.77 % i(6), minor (1, 3, 6) 2.000 0.033684 3.37 %
17 ii65(9), major (4, 6, 1, 2) 4.0 0.015717 1.57 % V7/iv, minor (1, #3, 5, 7) 2.000 0.033684 3.37 %
18 v7(b62), major (6, b7, b3, 4) 4.0 0.015717 1.57 % v6, minor (7, 2, 5) 2.000 0.033684 3.37 %
19 V, major (5, 7, 2) 4.0 0.015717 1.57 % iv, minor (4, 6, 1) 1.000 0.016842 1.68 %
20 V7(+964), major (5, 1, 3, 4) 4.0 0.015717 1.57 % NaN NaN NaN NaN NaN
21 V7(9^2), major (5, 6, 2, 4) 4.0 0.015717 1.57 % NaN NaN NaN NaN NaN
22 iM7(^4), major (1, b3, 4, 7) 4.0 0.015717 1.57 % NaN NaN NaN NaN NaN
23 IV, major (4, 6, 1) 4.0 0.015717 1.57 % NaN NaN NaN NaN NaN
24 I+7(#9), major (1, 3, #5, b7) 4.0 0.015717 1.57 % NaN NaN NaN NaN NaN
25 V7/V, major (2, #4, 6, 1) 3.5 0.013752 1.38 % NaN NaN NaN NaN NaN
26 ii(9), major (2, 4, 6) 3.0 0.011788 1.18 % NaN NaN NaN NaN NaN
27 V7(9)/V, major (2, #4, 6, 1) 3.0 0.011788 1.18 % NaN NaN NaN NaN NaN
28 i, major (1, b3, 5) 3.0 0.011788 1.18 % NaN NaN NaN NaN NaN
29 ii(139), major (2, 4, 6) 3.0 0.011788 1.18 % NaN NaN NaN NaN NaN
30 ii6(119), major (4, 6, 2) 3.0 0.011788 1.18 % NaN NaN NaN NaN NaN
31 V(6^2), major (5, 6, 3) 3.0 0.011788 1.18 % NaN NaN NaN NaN NaN
32 V(64), major (5, 1, 3) 3.0 0.011788 1.18 % NaN NaN NaN NaN NaN
33 IVM65, major (6, 1, 3, 4) 2.5 0.009823 0.98 % NaN NaN NaN NaN NaN
34 V7, major (5, 7, 2, 4) 2.0 0.007859 0.79 % NaN NaN NaN NaN NaN
35 VM7(^2), major (5, 6, 2, #4) 2.0 0.007859 0.79 % NaN NaN NaN NaN NaN
36 iM7(4), major (1, 4, 5, 7) 2.0 0.007859 0.79 % NaN NaN NaN NaN NaN
37 iM7, major (1, b3, 5, 7) 2.0 0.007859 0.79 % NaN NaN NaN NaN NaN
38 iii7(b9), major (3, 5, 7, 2) 2.0 0.007859 0.79 % NaN NaN NaN NaN NaN
39 ii(64+2), major (2, 5, 7) 2.0 0.007859 0.79 % NaN NaN NaN NaN NaN
40 v(b96^2), major (5, 6, 3) 2.0 0.007859 0.79 % NaN NaN NaN NaN NaN
41 v(#119^2), major (5, 6, 2) 2.0 0.007859 0.79 % NaN NaN NaN NaN NaN
42 V7(6), major (5, 7, 3, 4) 2.0 0.007859 0.79 % NaN NaN NaN NaN NaN
43 V(b6^2), major (5, 6, b3) 2.0 0.007859 0.79 % NaN NaN NaN NaN NaN
44 V(b139), major (5, 7, 2) 2.0 0.007859 0.79 % NaN NaN NaN NaN NaN
45 ii7(+2), major (2, 4, 6, 1) 2.0 0.007859 0.79 % NaN NaN NaN NaN NaN
46 ii(94), major (2, 5, 6) 2.0 0.007859 0.79 % NaN NaN NaN NaN NaN
47 IM7(6), major (1, 3, 6, 7) 2.0 0.007859 0.79 % NaN NaN NaN NaN NaN
48 ii(13), major (2, 4, 6) 1.0 0.003929 0.39 % NaN NaN NaN NaN NaN
49 V7(964), major (5, 1, 3, 4) 1.0 0.003929 0.39 % NaN NaN NaN NaN NaN
50 i(b13), major (1, b3, 5) 1.0 0.003929 0.39 % NaN NaN NaN NaN NaN
51 ii2, major (1, 2, 4, 6) 1.0 0.003929 0.39 % NaN NaN NaN NaN NaN
52 I(^2), major (1, 2, 5) 1.0 0.003929 0.39 % NaN NaN NaN NaN NaN
53 iii64, major (7, 3, 5) 1.0 0.003929 0.39 % NaN NaN NaN NaN NaN
54 ii64(9), major (6, 2, 4) 0.5 0.001965 0.2 % NaN NaN NaN NaN NaN
55 iii, major (3, 5, 7) 0.5 0.001965 0.2 % NaN NaN NaN NaN NaN
56 iii43/V, major (#4, 6, 7, 2) 0.5 0.001965 0.2 % NaN NaN NaN NaN NaN
57 IV64(9), major (1, 4, 6) 0.5 0.001965 0.2 % NaN NaN NaN NaN NaN
58 I64(9), major (5, 1, 3) 0.5 0.001965 0.2 % NaN NaN NaN NaN NaN
59 vi, major (6, 1, 3) 0.5 0.001965 0.2 % NaN NaN NaN NaN NaN
chords_by_mode.apply_step("Counter")
count
mode corpus piece chord_and_mode scale_degrees
major poulenc_mouvements_perpetuels 01_assez_modere IM7, major (1, 3, 5, 7) 13
I, major (1, 3, 5) 12
V(^2), major (5, 6, 2) 11
V7(^2), major (5, 6, 2, 4) 9
i7, major (1, b3, 5, b7) 6
... ... ... ... ... ...
minor poulenc_mouvements_perpetuels 02_tres_modere i(96), minor (1, 3, 6) 2
iio6, minor (4, 6, 2) 2
v6, minor (7, 2, 5) 2
iv, minor (4, 6, 1) 1
V7(9b6)/III, minor (7, 2, b5, 6) 1

79 rows × 1 columns

chords_by_mode.format = "scale_degree"
chords_by_mode.get_default_analysis().make_ranking_table()
mode major minor
chord_and_mode scale_degrees duration_qb proportion proportion_% chord_and_mode scale_degrees duration_qb proportion proportion_%
rank
1 ii7(9), major (2, 4, 6, 1) 20.5 0.080550 8.06 % iv(96), minor (4, 6, 2) 7.000 0.117895 11.79 %
2 I, major (1, 3, 5) 18.0 0.070727 7.07 % i2, minor (7, 1, 3, 5) 5.000 0.084211 8.42 %
3 iii(b9), major (3, 5, 7) 16.0 0.062868 6.29 % i, minor (1, 3, 5) 5.000 0.084211 8.42 %
4 IVM2, major (3, 4, 6, 1) 14.0 0.055010 5.5 % i64(13), minor (5, 1, 3) 4.000 0.067368 6.74 %
5 IM7, major (1, 3, 5, 7) 13.0 0.051081 5.11 % v, minor (5, 7, 2) 4.000 0.067368 6.74 %
6 V(^2), major (5, 6, 2) 11.0 0.043222 4.32 % ii%65, minor (4, 6, 1, 2) 4.000 0.067368 6.74 %
7 V7(^2), major (5, 6, 2, 4) 9.0 0.035363 3.54 % i(9b6), minor (1, 3, b6) 4.000 0.067368 6.74 %
8 V(96), major (5, 7, 3) 8.0 0.031434 3.14 % ii%43, minor (6, 1, 2, 4) 4.000 0.067368 6.74 %
9 ii7, major (2, 4, 6, 1) 8.0 0.031434 3.14 % i43, minor (5, 7, 1, 3) 3.000 0.050526 5.05 %
10 V(6), major (5, 7, 3) 7.0 0.027505 2.75 % V7(9b6)/III, minor (7, 2, b5, 6) 2.375 0.040000 4.0 %
11 ii, major (2, 4, 6) 6.5 0.025540 2.55 % iio6, minor (4, 6, 2) 2.000 0.033684 3.37 %
12 IM7(9), major (1, 3, 5, 7) 6.0 0.023576 2.36 % V(9)/III, minor (7, 2, 4) 2.000 0.033684 3.37 %
13 i7, major (1, b3, 5, b7) 6.0 0.023576 2.36 % V(96)/III, minor (7, 2, 5) 2.000 0.033684 3.37 %
14 V(6+4), major (5, 7, 3) 6.0 0.023576 2.36 % i(96), minor (1, 3, 6) 2.000 0.033684 3.37 %
15 v(119), major (5, b7, 2) 5.0 0.019646 1.96 % i(9), minor (1, 3, 5) 2.000 0.033684 3.37 %
16 V7(96), major (5, 7, 3, 4) 4.5 0.017682 1.77 % i(6), minor (1, 3, 6) 2.000 0.033684 3.37 %
17 ii65(9), major (4, 6, 1, 2) 4.0 0.015717 1.57 % V7/iv, minor (1, #3, 5, 7) 2.000 0.033684 3.37 %
18 v7(b62), major (6, b7, b3, 4) 4.0 0.015717 1.57 % v6, minor (7, 2, 5) 2.000 0.033684 3.37 %
19 V, major (5, 7, 2) 4.0 0.015717 1.57 % iv, minor (4, 6, 1) 1.000 0.016842 1.68 %
20 V7(+964), major (5, 1, 3, 4) 4.0 0.015717 1.57 % NaN NaN NaN NaN NaN
21 V7(9^2), major (5, 6, 2, 4) 4.0 0.015717 1.57 % NaN NaN NaN NaN NaN
22 iM7(^4), major (1, b3, 4, 7) 4.0 0.015717 1.57 % NaN NaN NaN NaN NaN
23 IV, major (4, 6, 1) 4.0 0.015717 1.57 % NaN NaN NaN NaN NaN
24 I+7(#9), major (1, 3, #5, b7) 4.0 0.015717 1.57 % NaN NaN NaN NaN NaN
25 V7/V, major (2, #4, 6, 1) 3.5 0.013752 1.38 % NaN NaN NaN NaN NaN
26 ii(9), major (2, 4, 6) 3.0 0.011788 1.18 % NaN NaN NaN NaN NaN
27 V7(9)/V, major (2, #4, 6, 1) 3.0 0.011788 1.18 % NaN NaN NaN NaN NaN
28 i, major (1, b3, 5) 3.0 0.011788 1.18 % NaN NaN NaN NaN NaN
29 ii(139), major (2, 4, 6) 3.0 0.011788 1.18 % NaN NaN NaN NaN NaN
30 ii6(119), major (4, 6, 2) 3.0 0.011788 1.18 % NaN NaN NaN NaN NaN
31 V(6^2), major (5, 6, 3) 3.0 0.011788 1.18 % NaN NaN NaN NaN NaN
32 V(64), major (5, 1, 3) 3.0 0.011788 1.18 % NaN NaN NaN NaN NaN
33 IVM65, major (6, 1, 3, 4) 2.5 0.009823 0.98 % NaN NaN NaN NaN NaN
34 V7, major (5, 7, 2, 4) 2.0 0.007859 0.79 % NaN NaN NaN NaN NaN
35 VM7(^2), major (5, 6, 2, #4) 2.0 0.007859 0.79 % NaN NaN NaN NaN NaN
36 iM7(4), major (1, 4, 5, 7) 2.0 0.007859 0.79 % NaN NaN NaN NaN NaN
37 iM7, major (1, b3, 5, 7) 2.0 0.007859 0.79 % NaN NaN NaN NaN NaN
38 iii7(b9), major (3, 5, 7, 2) 2.0 0.007859 0.79 % NaN NaN NaN NaN NaN
39 ii(64+2), major (2, 5, 7) 2.0 0.007859 0.79 % NaN NaN NaN NaN NaN
40 v(b96^2), major (5, 6, 3) 2.0 0.007859 0.79 % NaN NaN NaN NaN NaN
41 v(#119^2), major (5, 6, 2) 2.0 0.007859 0.79 % NaN NaN NaN NaN NaN
42 V7(6), major (5, 7, 3, 4) 2.0 0.007859 0.79 % NaN NaN NaN NaN NaN
43 V(b6^2), major (5, 6, b3) 2.0 0.007859 0.79 % NaN NaN NaN NaN NaN
44 V(b139), major (5, 7, 2) 2.0 0.007859 0.79 % NaN NaN NaN NaN NaN
45 ii7(+2), major (2, 4, 6, 1) 2.0 0.007859 0.79 % NaN NaN NaN NaN NaN
46 ii(94), major (2, 5, 6) 2.0 0.007859 0.79 % NaN NaN NaN NaN NaN
47 IM7(6), major (1, 3, 6, 7) 2.0 0.007859 0.79 % NaN NaN NaN NaN NaN
48 ii(13), major (2, 4, 6) 1.0 0.003929 0.39 % NaN NaN NaN NaN NaN
49 V7(964), major (5, 1, 3, 4) 1.0 0.003929 0.39 % NaN NaN NaN NaN NaN
50 i(b13), major (1, b3, 5) 1.0 0.003929 0.39 % NaN NaN NaN NaN NaN
51 ii2, major (1, 2, 4, 6) 1.0 0.003929 0.39 % NaN NaN NaN NaN NaN
52 I(^2), major (1, 2, 5) 1.0 0.003929 0.39 % NaN NaN NaN NaN NaN
53 iii64, major (7, 3, 5) 1.0 0.003929 0.39 % NaN NaN NaN NaN NaN
54 ii64(9), major (6, 2, 4) 0.5 0.001965 0.2 % NaN NaN NaN NaN NaN
55 iii, major (3, 5, 7) 0.5 0.001965 0.2 % NaN NaN NaN NaN NaN
56 iii43/V, major (#4, 6, 7, 2) 0.5 0.001965 0.2 % NaN NaN NaN NaN NaN
57 IV64(9), major (1, 4, 6) 0.5 0.001965 0.2 % NaN NaN NaN NaN NaN
58 I64(9), major (5, 1, 3) 0.5 0.001965 0.2 % NaN NaN NaN NaN NaN
59 vi, major (6, 1, 3) 0.5 0.001965 0.2 % NaN NaN NaN NaN NaN
unigram_proportions.plot_grouped()